{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# `m_u_parameters_chart`\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "tags": [
          "hide_input"
        ]
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "  #altair-viz-c61f2ef36f2647638a7ad0aa23688736.vega-embed {\n",
              "    width: 100%;\n",
              "    display: flex;\n",
              "  }\n",
              "\n",
              "  #altair-viz-c61f2ef36f2647638a7ad0aa23688736.vega-embed details,\n",
              "  #altair-viz-c61f2ef36f2647638a7ad0aa23688736.vega-embed details summary {\n",
              "    position: relative;\n",
              "  }\n",
              "</style>\n",
              "<div id=\"altair-viz-c61f2ef36f2647638a7ad0aa23688736\"></div>\n",
              "<script type=\"text/javascript\">\n",
              "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
              "  (function(spec, embedOpt){\n",
              "    let outputDiv = document.currentScript.previousElementSibling;\n",
              "    if (outputDiv.id !== \"altair-viz-c61f2ef36f2647638a7ad0aa23688736\") {\n",
              "      outputDiv = document.getElementById(\"altair-viz-c61f2ef36f2647638a7ad0aa23688736\");\n",
              "    }\n",
              "    const paths = {\n",
              "      \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
              "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
              "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.17.0?noext\",\n",
              "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
              "    };\n",
              "\n",
              "    function maybeLoadScript(lib, version) {\n",
              "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
              "      return (VEGA_DEBUG[key] == version) ?\n",
              "        Promise.resolve(paths[lib]) :\n",
              "        new Promise(function(resolve, reject) {\n",
              "          var s = document.createElement('script');\n",
              "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
              "          s.async = true;\n",
              "          s.onload = () => {\n",
              "            VEGA_DEBUG[key] = version;\n",
              "            return resolve(paths[lib]);\n",
              "          };\n",
              "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
              "          s.src = paths[lib];\n",
              "        });\n",
              "    }\n",
              "\n",
              "    function showError(err) {\n",
              "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
              "      throw err;\n",
              "    }\n",
              "\n",
              "    function displayChart(vegaEmbed) {\n",
              "      vegaEmbed(outputDiv, spec, embedOpt)\n",
              "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
              "    }\n",
              "\n",
              "    if(typeof define === \"function\" && define.amd) {\n",
              "      requirejs.config({paths});\n",
              "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
              "    } else {\n",
              "      maybeLoadScript(\"vega\", \"5\")\n",
              "        .then(() => maybeLoadScript(\"vega-lite\", \"5.17.0\"))\n",
              "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
              "        .catch(showError)\n",
              "        .then(() => displayChart(vegaEmbed));\n",
              "    }\n",
              "  })({\"config\": {\"view\": {\"continuousWidth\": 300, \"continuousHeight\": 300, \"discreteHeight\": 300, \"discreteWidth\": 400}, \"header\": {\"title\": null}, \"title\": {\"anchor\": \"middle\", \"offset\": 10}}, \"hconcat\": [{\"mark\": \"bar\", \"encoding\": {\"color\": {\"value\": \"green\"}, \"row\": {\"field\": \"comparison_name\", \"header\": {\"labelAlign\": \"left\", \"labelAnchor\": \"middle\", \"labelAngle\": 0}, \"sort\": {\"field\": \"comparison_sort_order\"}, \"type\": \"nominal\"}, \"tooltip\": [{\"field\": \"comparison_name\", \"title\": \"Comparison name\", \"type\": \"nominal\"}, {\"field\": \"label_for_charts\", \"title\": \"Label\", \"type\": \"ordinal\"}, {\"field\": \"sql_condition\", \"title\": \"SQL condition\", \"type\": \"nominal\"}, {\"field\": \"m_probability\", \"format\": \".10~g\", \"title\": \"M probability\", \"type\": \"quantitative\"}, {\"field\": \"u_probability\", \"format\": \".10~g\", \"title\": \"U probability\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor\", \"format\": \",.6f\", \"title\": \"Bayes factor = m/u\", \"type\": \"quantitative\"}, {\"field\": \"log2_bayes_factor\", \"format\": \".4~g\", \"title\": \"Match weight = log2(m/u)\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor_description\", \"title\": \"Match weight description\", \"type\": \"nominal\"}, {\"field\": \"m_probability_description\", \"title\": \"m probability description\", \"type\": \"nominal\"}, {\"field\": \"u_probability_description\", \"title\": \"u probability description\", \"type\": \"nominal\"}], \"x\": {\"axis\": {\"title\": \"Proportion of record comparisons\"}, \"field\": \"m_probability\", \"type\": \"quantitative\"}, \"y\": {\"axis\": {\"title\": null}, \"field\": \"label_for_charts\", \"sort\": {\"field\": \"comparison_vector_value\", \"order\": \"descending\"}, \"type\": \"nominal\"}}, \"height\": {\"step\": 12}, \"resolve\": {\"scale\": {\"y\": \"independent\"}}, \"title\": {\"text\": \"Amongst matching record comparisons:\", \"fontSize\": 12, \"fontWeight\": \"bold\"}, \"transform\": [{\"filter\": \"(datum.bayes_factor != 'no-op filter due to vega lite issue 4680')\"}], \"width\": 150}, {\"mark\": \"bar\", \"encoding\": {\"color\": {\"value\": \"red\"}, \"row\": {\"field\": \"comparison_name\", \"header\": {\"labels\": false}, \"sort\": {\"field\": \"comparison_sort_order\"}, \"type\": \"nominal\"}, \"tooltip\": [{\"field\": \"comparison_name\", \"title\": \"Comparison name\", \"type\": \"nominal\"}, {\"field\": \"label_for_charts\", \"title\": \"Label\", \"type\": \"ordinal\"}, {\"field\": \"sql_condition\", \"title\": \"SQL condition\", \"type\": \"nominal\"}, {\"field\": \"m_probability\", \"format\": \".10~g\", \"title\": \"M probability\", \"type\": \"quantitative\"}, {\"field\": \"u_probability\", \"format\": \".10~g\", \"title\": \"U probability\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor\", \"format\": \",.6f\", \"title\": \"Bayes factor = m/u\", \"type\": \"quantitative\"}, {\"field\": \"log2_bayes_factor\", \"format\": \".4~g\", \"title\": \"Match weight = log2(m/u)\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor_description\", \"title\": \"Match weight description\", \"type\": \"nominal\"}, {\"field\": \"m_probability_description\", \"title\": \"m probability description\", \"type\": \"nominal\"}, {\"field\": \"u_probability_description\", \"title\": \"u probability description\", \"type\": \"nominal\"}], \"x\": {\"axis\": {\"title\": \"Proportion of record comparisons\"}, \"field\": \"u_probability\", \"type\": \"quantitative\"}, \"y\": {\"axis\": {\"title\": null}, \"field\": \"label_for_charts\", \"sort\": {\"field\": \"comparison_vector_value\", \"order\": \"descending\"}, \"type\": \"nominal\"}}, \"height\": {\"step\": 12}, \"resolve\": {\"scale\": {\"y\": \"independent\"}}, \"title\": {\"text\": \"Amongst non-matching record comparisons:\", \"fontSize\": 12, \"fontWeight\": \"bold\"}, \"transform\": [{\"filter\": \"(datum.bayes_factor != 'no-op filter2 due to vega lite issue 4680')\"}], \"width\": 150}], \"data\": {\"name\": \"data-0317940c67660b1599eba58da0e7c816\"}, \"title\": {\"text\": \"Proportion of record comparisons in each comparison level by match status\", \"subtitle\": \"(m and u probabilities)\"}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.9.3.json\", \"datasets\": {\"data-0317940c67660b1599eba58da0e7c816\": [{\"comparison_name\": \"first_name\", \"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Exact match on first_name\", \"m_probability\": 0.49136441060423774, \"u_probability\": 0.0057935713975033705, \"m_probability_description\": \"Amongst matching record comparisons, 49.14% of records (i.e. one in 2.035) are in the exact match on first_name comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.5794% of records (i.e. one in 173) are in the exact match on first_name comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 84.81200573725248, \"log2_bayes_factor\": 6.406196597784454, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on first_name` then comparison is 84.81 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"jaro_winkler_similarity(\\\"first_name_l\\\", \\\"first_name_r\\\") >= 0.9\", \"label_for_charts\": \"Jaro-Winkler distance of first_name >= 0.9\", \"m_probability\": 0.19147351818420014, \"u_probability\": 0.003386832528670639, \"m_probability_description\": \"Amongst matching record comparisons, 19.15% of records (i.e. one in 5.223) are in the jaro-winkler distance of first_name >= 0.9 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.3387% of records (i.e. one in 295) are in the jaro-winkler distance of first_name >= 0.9 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 56.53468737037173, \"log2_bayes_factor\": 5.821064412712824, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of first_name >= 0.9` then comparison is 56.53 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"jaro_winkler_similarity(\\\"first_name_l\\\", \\\"first_name_r\\\") >= 0.7\", \"label_for_charts\": \"Jaro-Winkler distance of first_name >= 0.7\", \"m_probability\": 0.11346321825427941, \"u_probability\": 0.019439490815246544, \"m_probability_description\": \"Amongst matching record comparisons, 11.35% of records (i.e. one in 8.813) are in the jaro-winkler distance of first_name >= 0.7 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 1.944% of records (i.e. one in 51.44) are in the jaro-winkler distance of first_name >= 0.7 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 5.836738180677517, \"log2_bayes_factor\": 2.5451623545127906, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of first_name >= 0.7` then comparison is 5.837 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.20369885295728274, \"u_probability\": 0.9713801052585794, \"m_probability_description\": \"Amongst matching record comparisons, 20.37% of records (i.e. one in 4.909) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 97.14% of records (i.e. one in 1.029) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.20970045799224857, \"log2_bayes_factor\": -2.253598082559084, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 4.769 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"surname\", \"sql_condition\": \"\\\"surname_l\\\" = \\\"surname_r\\\"\", \"label_for_charts\": \"Exact match on surname\", \"m_probability\": 0.4345244833248351, \"u_probability\": 0.004889975550122249, \"m_probability_description\": \"Amongst matching record comparisons, 43.45% of records (i.e. one in 2.301) are in the exact match on surname comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.489% of records (i.e. one in 204) are in the exact match on surname comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 88.86025683992878, \"log2_bayes_factor\": 6.473466406178517, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on surname` then comparison is 88.86 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"surname\", \"sql_condition\": \"jaro_similarity(\\\"surname_l\\\", \\\"surname_r\\\") >= 0.9\", \"label_for_charts\": \"Jaro distance of 'surname >= 0.9'\", \"m_probability\": 0.2163740833960705, \"u_probability\": 0.0025524597651737017, \"m_probability_description\": \"Amongst matching record comparisons, 21.64% of records (i.e. one in 4.622) are in the jaro distance of 'surname >= 0.9' comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.2552% of records (i.e. one in 392) are in the jaro distance of 'surname >= 0.9' comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 84.77081063071945, \"log2_bayes_factor\": 6.405495677982012, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro distance of 'surname >= 0.9'` then comparison is 84.77 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"surname\", \"sql_condition\": \"jaro_similarity(\\\"surname_l\\\", \\\"surname_r\\\") >= 0.7\", \"label_for_charts\": \"Jaro distance of 'surname >= 0.7'\", \"m_probability\": 0.13100004234324159, \"u_probability\": 0.01614766651441468, \"m_probability_description\": \"Amongst matching record comparisons, 13.1% of records (i.e. one in 7.634) are in the jaro distance of 'surname >= 0.7' comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 1.615% of records (i.e. one in 61.93) are in the jaro distance of 'surname >= 0.7' comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 8.112629910105007, \"log2_bayes_factor\": 3.0201696756280945, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro distance of 'surname >= 0.7'` then comparison is 8.113 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"surname\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.21810139093585287, \"u_probability\": 0.9764098981702893, \"m_probability_description\": \"Amongst matching record comparisons, 21.81% of records (i.e. one in 4.585) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 97.64% of records (i.e. one in 1.024) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.22337072918305792, \"log2_bayes_factor\": -2.162487949483275, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 4.477 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"dob\", \"sql_condition\": \"\\\"dob_l\\\" = \\\"dob_r\\\"\", \"label_for_charts\": \"Exact match on date of birth\", \"m_probability\": 0.389983939276255, \"u_probability\": 0.0017477477477477479, \"m_probability_description\": \"Amongst matching record comparisons, 39% of records (i.e. one in 2.564) are in the exact match on date of birth comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.1748% of records (i.e. one in 572) are in the exact match on date of birth comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 223.13514051373352, \"log2_bayes_factor\": 7.801773924569989, \"comparison_vector_value\": 4, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `exact match on date of birth` then comparison is 223 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"damerau_levenshtein(\\\"dob_l\\\", \\\"dob_r\\\") <= 1\", \"label_for_charts\": \"DamerauLevenshtein distance <= 1\", \"m_probability\": 0.14884650150455297, \"u_probability\": 0.0016436436436436436, \"m_probability_description\": \"Amongst matching record comparisons, 14.88% of records (i.e. one in 6.718) are in the dameraulevenshtein distance <= 1 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.1644% of records (i.e. one in 608) are in the dameraulevenshtein distance <= 1 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 90.55886419186872, \"log2_bayes_factor\": 6.500783958589023, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `dameraulevenshtein distance <= 1` then comparison is 90.56 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"ABS(EPOCH(try_strptime(\\\"dob_l\\\", '%Y-%m-%d')) - EPOCH(try_strptime(\\\"dob_r\\\", '%Y-%m-%d'))) <= 31557600.0\", \"label_for_charts\": \"Abs date difference <= 1 year\", \"m_probability\": 0.19399707416347944, \"u_probability\": 0.03546146146146146, \"m_probability_description\": \"Amongst matching record comparisons, 19.4% of records (i.e. one in 5.155) are in the abs date difference <= 1 year comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 3.546% of records (i.e. one in 28.2) are in the abs date difference <= 1 year comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 5.470645206608592, \"log2_bayes_factor\": 2.4517109941661124, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `abs date difference <= 1 year` then comparison is 5.471 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"ABS(EPOCH(try_strptime(\\\"dob_l\\\", '%Y-%m-%d')) - EPOCH(try_strptime(\\\"dob_r\\\", '%Y-%m-%d'))) <= 2629800.0\", \"label_for_charts\": \"Abs date difference <= 1 month\", \"m_probability\": 0.012500000000000011, \"u_probability\": 0.06299605249474372, \"m_probability_description\": \"Amongst matching record comparisons, 1.25% of records (i.e. one in 80) are in the abs date difference <= 1 month comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 6.3% of records (i.e. one in 15.87) are in the abs date difference <= 1 month comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.19842513149602492, \"log2_bayes_factor\": -2.3333333333333335, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `abs date difference <= 1 month` then comparison is 5.04 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.26717248505571256, \"u_probability\": 0.9611471471471471, \"m_probability_description\": \"Amongst matching record comparisons, 26.72% of records (i.e. one in 3.743) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 96.11% of records (i.e. one in 1.04) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.2779725100872715, \"log2_bayes_factor\": -1.846985879283028, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 3.597 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"city\", \"sql_condition\": \"\\\"city_l\\\" = \\\"city_r\\\"\", \"label_for_charts\": \"Exact match on city\", \"m_probability\": 0.5611221596314007, \"u_probability\": 0.0551475711801453, \"m_probability_description\": \"Amongst matching record comparisons, 56.11% of records (i.e. one in 1.782) are in the exact match on city comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 5.515% of records (i.e. one in 18.13) are in the exact match on city comparison level\", \"has_tf_adjustments\": true, \"tf_adjustment_column\": \"city\", \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 10.17492062884214, \"log2_bayes_factor\": 3.34694563544056, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 1, \"bayes_factor_description\": \"If comparison level is `exact match on city` then comparison is 10.17 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 3}, {\"comparison_name\": \"city\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4388778403685992, \"u_probability\": 0.9448524288198547, \"m_probability_description\": \"Amongst matching record comparisons, 43.89% of records (i.e. one in 2.279) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 94.49% of records (i.e. one in 1.058) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.4644935303989948, \"log2_bayes_factor\": -1.1062695924156825, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 1, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 2.153 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 3}, {\"comparison_name\": \"email\", \"sql_condition\": \"\\\"email_l\\\" = \\\"email_r\\\"\", \"label_for_charts\": \"Exact match on email\", \"m_probability\": 0.552153409103165, \"u_probability\": 0.0021938713143283602, \"m_probability_description\": \"Amongst matching record comparisons, 55.22% of records (i.e. one in 1.811) are in the exact match on email comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.2194% of records (i.e. one in 456) are in the exact match on email comparison level\", \"has_tf_adjustments\": true, \"tf_adjustment_column\": \"email\", \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 251.67994380390687, \"log2_bayes_factor\": 7.975446443510344, \"comparison_vector_value\": 4, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `exact match on email` then comparison is 252 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"NULLIF(regexp_extract(\\\"email_l\\\", '^[^@]+', 0), '') = NULLIF(regexp_extract(\\\"email_r\\\", '^[^@]+', 0), '')\", \"label_for_charts\": \"Exact match on username\", \"m_probability\": 0.22055262276218585, \"u_probability\": 0.0010390328952024346, \"m_probability_description\": \"Amongst matching record comparisons, 22.06% of records (i.e. one in 4.534) are in the exact match on username comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.1039% of records (i.e. one in 962) are in the exact match on username comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 212.2672186612683, \"log2_bayes_factor\": 7.72973777662564, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `exact match on username` then comparison is 212 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"jaro_winkler_similarity(\\\"email_l\\\", \\\"email_r\\\") >= 0.88\", \"label_for_charts\": \"Jaro-Winkler distance of email >= 0.88\", \"m_probability\": 0.2138389403844258, \"u_probability\": 0.0009135769109519858, \"m_probability_description\": \"Amongst matching record comparisons, 21.38% of records (i.e. one in 4.676) are in the jaro-winkler distance of email >= 0.88 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.09136% of records (i.e. one in 1,095) are in the jaro-winkler distance of email >= 0.88 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 234.0678029631863, \"log2_bayes_factor\": 7.870782688940884, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of email >= 0.88` then comparison is 234 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"jaro_winkler_similarity(NULLIF(regexp_extract(\\\"email_l\\\", '^[^@]+', 0), ''), NULLIF(regexp_extract(\\\"email_r\\\", '^[^@]+', 0), '')) >= 0.88\", \"label_for_charts\": \"Jaro-Winkler >0.88 on username\", \"m_probability\": 0.012500000000000011, \"u_probability\": 0.000501823937001795, \"m_probability_description\": \"Amongst matching record comparisons, 1.25% of records (i.e. one in 80) are in the jaro-winkler >0.88 on username comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.05018% of records (i.e. one in 1,993) are in the jaro-winkler >0.88 on username comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 24.909134615384634, \"log2_bayes_factor\": 4.638602995720225, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `jaro-winkler >0.88 on username` then comparison is 24.91 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.01345502775022321, \"u_probability\": 0.9953516949425154, \"m_probability_description\": \"Amongst matching record comparisons, 1.346% of records (i.e. one in 74.32) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 99.54% of records (i.e. one in 1.005) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.013517862900710968, \"log2_bayes_factor\": -6.2089891022264005, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 73.98 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}]}}, {\"mode\": \"vega-lite\"});\n",
              "</script>"
            ],
            "text/plain": [
              "alt.HConcatChart(...)"
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "chart"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n",
        "!!! info \"At a glance\"\n",
        "    **Useful for:** Looking at the m and u values generated by a Splink model.\n",
        "\n",
        "    **API Documentation:** [m_u_parameters_chart()](../api_docs/visualisations.md#splink.internals.linker_components.visualisations.LinkerVisualisations.m_u_parameters_chart)\n",
        "\n",
        "    **What is needed to generate the chart?** A trained Splink model."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### What the chart shows\n",
        "\n",
        "The `m_u_parameters_chart` shows the results of a trained Splink model:\n",
        "\n",
        "- The left chart shows the estimated m probabilities from the Splink model \n",
        "- The right chart shows the estimated u probabilities from the Splink model.\n",
        "\n",
        "Each comparison within a model is represented in trained m and u values that have been estimated during the Splink model training for each comparison level.\n",
        "\n",
        "??? note \"What the chart tooltip shows\"\n",
        "\n",
        "    #### Estimated m probability tooltip\n",
        "\n",
        "    ![](./img/m_u_parameters_chart_tooltip_1.png)\n",
        "\n",
        "    The tooltip of the left chart shows information based on the comparison level bar that the user is hovering over, including:\n",
        "\n",
        "    - An explanation of the m probability for the comparison level.\n",
        "    - The name of the comparison and comparison level.\n",
        "    - The comparison level condition as an SQL statement.\n",
        "    - The m and u proability for the comparison level.\n",
        "    - The resulting bayes factor and match weight for the comparison level.\n",
        "\n",
        "    #### Estimated u probability tooltip\n",
        "\n",
        "    ![](./img/m_u_parameters_chart_tooltip_2.png)\n",
        "\n",
        "    The tooltip of the right chart shows information based on the comparison level bar that the user is hovering over, including:\n",
        "\n",
        "    - An explanation of the u probability from the comparison level.\n",
        "    - The name of the comparison and comparison level.\n",
        "    - The comparison level condition as an SQL statement.\n",
        "    - The m and u proability for the comparison level.\n",
        "    - The resulting bayes factor and match weight for the comparison level."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### How to interpret the chart\n",
        "\n",
        "Each bar of the left chart shows the probability of a given comparison level when two records are a match. This can also be interpreted as the proportion of matching records which are allocated to the comparison level (as stated in the x axis label).\n",
        "\n",
        "Similarly, each bar of the right chart shows the probability of a given comparison level when two records are not a match. This can also be interpreted as the proportion of non-matching records which are allocated to the comparison level (as stated in the x axis label).\n",
        "\n",
        "!!! note \"Further Reading\"\n",
        "\n",
        "    For a more comprehensive introduction to m and u probabilities, check out the [Fellegi Sunter model topic guide.](../topic_guides/theory/fellegi_sunter.md#parameters-of-the-fellegi-sunter-model)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Actions to take as a result of the chart\n",
        "\n",
        "As with the `match_weights_chart`, one of the most effective methods to assess a Splink model is to walk through each of the comparison levels of the `m_u_parameters_chart` and sense check the m and u probabilities that have been allocated by the model.\n",
        "\n",
        "For example, for all non-matching pairwise comparisons (which form the vast majority of all pairwise comparisons), it makes sense that the exact match and fuzzy levels occur very rarely. Furthermore, `dob` and `city` are lower cardinality features (i.e. have fewer possible values) than names so \"All other comparisons\" is less likely.\n",
        "\n",
        "If there are any m or u values that appear unusual, check out the values generated for each training session in the [`parameter_estimate_comparisons_chart`](./parameter_estimate_comparisons_chart.ipynb)."
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Related Charts\n",
        "\n",
        "::cards::\n",
        "[\n",
        "    {\n",
        "    \"title\": \"`match weights chart`\",\n",
        "    \"image\": \"./img/match_weights_chart.png\",\n",
        "    \"url\": \"./match_weights_chart.ipynb\"\n",
        "    },\n",
        "    {\n",
        "    \"title\": \"`parameter estimate comparisons chart`\",\n",
        "    \"image\": \"./img/parameter_estimate_comparisons_chart.png\",\n",
        "    \"url\": \"./parameter_estimate_comparisons_chart.ipynb\"\n",
        "    },\n",
        "]\n",
        "::/cards::"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Worked Example"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "tags": [
          "hide_output"
        ]
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "You are using the default value for `max_pairs`, which may be too small and thus lead to inaccurate estimates for your model's u-parameters. Consider increasing to 1e8 or 1e9, which will result in more accurate estimates, but with a longer run time.\n",
            "----- Estimating u probabilities using random sampling -----\n",
            "u probability not trained for dob - Abs date difference <= 1 month (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "\n",
            "Estimated u probabilities using random sampling\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - first_name (no m values are trained).\n",
            "    - surname (no m values are trained).\n",
            "    - dob (some u values are not trained, no m values are trained).\n",
            "    - city (no m values are trained).\n",
            "    - email (no m values are trained).\n",
            "\n",
            "----- Starting EM training session -----\n",
            "\n",
            "Estimating the m probabilities of the model by blocking on:\n",
            "(l.\"first_name\" = r.\"first_name\") AND (l.\"surname\" = r.\"surname\")\n",
            "\n",
            "Parameter estimates will be made for the following comparison(s):\n",
            "    - dob\n",
            "    - city\n",
            "    - email\n",
            "\n",
            "Parameter estimates cannot be made for the following comparison(s) since they are used in the blocking rules: \n",
            "    - first_name\n",
            "    - surname\n",
            "\n",
            "WARNING:\n",
            "Level Abs date difference <= 1 month on comparison dob not observed in dataset, unable to train m value\n",
            "\n",
            "WARNING:\n",
            "Level Jaro-Winkler >0.88 on username on comparison email not observed in dataset, unable to train m value\n",
            "\n",
            "Iteration 1: Largest change in params was -0.463 in the m_probability of dob, level `Exact match on date of birth`\n",
            "Iteration 2: Largest change in params was 0.144 in probability_two_random_records_match\n",
            "Iteration 3: Largest change in params was 0.0328 in probability_two_random_records_match\n",
            "Iteration 4: Largest change in params was 0.0108 in probability_two_random_records_match\n",
            "Iteration 5: Largest change in params was 0.00444 in probability_two_random_records_match\n",
            "Iteration 6: Largest change in params was 0.00212 in probability_two_random_records_match\n",
            "Iteration 7: Largest change in params was 0.00111 in probability_two_random_records_match\n",
            "Iteration 8: Largest change in params was 0.000611 in probability_two_random_records_match\n",
            "Iteration 9: Largest change in params was 0.000347 in probability_two_random_records_match\n",
            "Iteration 10: Largest change in params was 0.0002 in probability_two_random_records_match\n",
            "Iteration 11: Largest change in params was 0.000117 in probability_two_random_records_match\n",
            "Iteration 12: Largest change in params was 6.85e-05 in probability_two_random_records_match\n",
            "\n",
            "EM converged after 12 iterations\n",
            "m probability not trained for dob - Abs date difference <= 1 month (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "m probability not trained for email - Jaro-Winkler >0.88 on username (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - first_name (no m values are trained).\n",
            "    - surname (no m values are trained).\n",
            "    - dob (some u values are not trained, some m values are not trained).\n",
            "    - email (some m values are not trained).\n",
            "\n",
            "----- Starting EM training session -----\n",
            "\n",
            "Estimating the m probabilities of the model by blocking on:\n",
            "l.\"dob\" = r.\"dob\"\n",
            "\n",
            "Parameter estimates will be made for the following comparison(s):\n",
            "    - first_name\n",
            "    - surname\n",
            "    - city\n",
            "    - email\n",
            "\n",
            "Parameter estimates cannot be made for the following comparison(s) since they are used in the blocking rules: \n",
            "    - dob\n",
            "\n",
            "WARNING:\n",
            "Level Jaro-Winkler >0.88 on username on comparison email not observed in dataset, unable to train m value\n",
            "\n",
            "Iteration 1: Largest change in params was 0.632 in probability_two_random_records_match\n",
            "Iteration 2: Largest change in params was 0.173 in probability_two_random_records_match\n",
            "Iteration 3: Largest change in params was 0.0865 in the m_probability of first_name, level `All other comparisons`\n",
            "Iteration 4: Largest change in params was 0.0354 in the m_probability of first_name, level `All other comparisons`\n",
            "Iteration 5: Largest change in params was 0.013 in probability_two_random_records_match\n",
            "Iteration 6: Largest change in params was 0.00552 in probability_two_random_records_match\n",
            "Iteration 7: Largest change in params was 0.00253 in probability_two_random_records_match\n",
            "Iteration 8: Largest change in params was 0.0012 in probability_two_random_records_match\n",
            "Iteration 9: Largest change in params was 0.000584 in probability_two_random_records_match\n",
            "Iteration 10: Largest change in params was 0.000286 in probability_two_random_records_match\n",
            "Iteration 11: Largest change in params was 0.000141 in probability_two_random_records_match\n",
            "Iteration 12: Largest change in params was 6.93e-05 in probability_two_random_records_match\n",
            "\n",
            "EM converged after 12 iterations\n",
            "m probability not trained for email - Jaro-Winkler >0.88 on username (comparison vector value: 1). This usually means the comparison level was never observed in the training data.\n",
            "\n",
            "Your model is not yet fully trained. Missing estimates for:\n",
            "    - dob (some u values are not trained, some m values are not trained).\n",
            "    - email (some m values are not trained).\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "\n",
              "<style>\n",
              "  #altair-viz-23384cc7f63843339598f52d72680e8c.vega-embed {\n",
              "    width: 100%;\n",
              "    display: flex;\n",
              "  }\n",
              "\n",
              "  #altair-viz-23384cc7f63843339598f52d72680e8c.vega-embed details,\n",
              "  #altair-viz-23384cc7f63843339598f52d72680e8c.vega-embed details summary {\n",
              "    position: relative;\n",
              "  }\n",
              "</style>\n",
              "<div id=\"altair-viz-23384cc7f63843339598f52d72680e8c\"></div>\n",
              "<script type=\"text/javascript\">\n",
              "  var VEGA_DEBUG = (typeof VEGA_DEBUG == \"undefined\") ? {} : VEGA_DEBUG;\n",
              "  (function(spec, embedOpt){\n",
              "    let outputDiv = document.currentScript.previousElementSibling;\n",
              "    if (outputDiv.id !== \"altair-viz-23384cc7f63843339598f52d72680e8c\") {\n",
              "      outputDiv = document.getElementById(\"altair-viz-23384cc7f63843339598f52d72680e8c\");\n",
              "    }\n",
              "    const paths = {\n",
              "      \"vega\": \"https://cdn.jsdelivr.net/npm/vega@5?noext\",\n",
              "      \"vega-lib\": \"https://cdn.jsdelivr.net/npm/vega-lib?noext\",\n",
              "      \"vega-lite\": \"https://cdn.jsdelivr.net/npm/vega-lite@5.17.0?noext\",\n",
              "      \"vega-embed\": \"https://cdn.jsdelivr.net/npm/vega-embed@6?noext\",\n",
              "    };\n",
              "\n",
              "    function maybeLoadScript(lib, version) {\n",
              "      var key = `${lib.replace(\"-\", \"\")}_version`;\n",
              "      return (VEGA_DEBUG[key] == version) ?\n",
              "        Promise.resolve(paths[lib]) :\n",
              "        new Promise(function(resolve, reject) {\n",
              "          var s = document.createElement('script');\n",
              "          document.getElementsByTagName(\"head\")[0].appendChild(s);\n",
              "          s.async = true;\n",
              "          s.onload = () => {\n",
              "            VEGA_DEBUG[key] = version;\n",
              "            return resolve(paths[lib]);\n",
              "          };\n",
              "          s.onerror = () => reject(`Error loading script: ${paths[lib]}`);\n",
              "          s.src = paths[lib];\n",
              "        });\n",
              "    }\n",
              "\n",
              "    function showError(err) {\n",
              "      outputDiv.innerHTML = `<div class=\"error\" style=\"color:red;\">${err}</div>`;\n",
              "      throw err;\n",
              "    }\n",
              "\n",
              "    function displayChart(vegaEmbed) {\n",
              "      vegaEmbed(outputDiv, spec, embedOpt)\n",
              "        .catch(err => showError(`Javascript Error: ${err.message}<br>This usually means there's a typo in your chart specification. See the javascript console for the full traceback.`));\n",
              "    }\n",
              "\n",
              "    if(typeof define === \"function\" && define.amd) {\n",
              "      requirejs.config({paths});\n",
              "      require([\"vega-embed\"], displayChart, err => showError(`Error loading script: ${err.message}`));\n",
              "    } else {\n",
              "      maybeLoadScript(\"vega\", \"5\")\n",
              "        .then(() => maybeLoadScript(\"vega-lite\", \"5.17.0\"))\n",
              "        .then(() => maybeLoadScript(\"vega-embed\", \"6\"))\n",
              "        .catch(showError)\n",
              "        .then(() => displayChart(vegaEmbed));\n",
              "    }\n",
              "  })({\"config\": {\"view\": {\"continuousWidth\": 300, \"continuousHeight\": 300, \"discreteHeight\": 300, \"discreteWidth\": 400}, \"header\": {\"title\": null}, \"title\": {\"anchor\": \"middle\", \"offset\": 10}}, \"hconcat\": [{\"mark\": \"bar\", \"encoding\": {\"color\": {\"value\": \"green\"}, \"row\": {\"field\": \"comparison_name\", \"header\": {\"labelAlign\": \"left\", \"labelAnchor\": \"middle\", \"labelAngle\": 0}, \"sort\": {\"field\": \"comparison_sort_order\"}, \"type\": \"nominal\"}, \"tooltip\": [{\"field\": \"comparison_name\", \"title\": \"Comparison name\", \"type\": \"nominal\"}, {\"field\": \"label_for_charts\", \"title\": \"Label\", \"type\": \"ordinal\"}, {\"field\": \"sql_condition\", \"title\": \"SQL condition\", \"type\": \"nominal\"}, {\"field\": \"m_probability\", \"format\": \".10~g\", \"title\": \"M probability\", \"type\": \"quantitative\"}, {\"field\": \"u_probability\", \"format\": \".10~g\", \"title\": \"U probability\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor\", \"format\": \",.6f\", \"title\": \"Bayes factor = m/u\", \"type\": \"quantitative\"}, {\"field\": \"log2_bayes_factor\", \"format\": \".4~g\", \"title\": \"Match weight = log2(m/u)\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor_description\", \"title\": \"Match weight description\", \"type\": \"nominal\"}, {\"field\": \"m_probability_description\", \"title\": \"m probability description\", \"type\": \"nominal\"}, {\"field\": \"u_probability_description\", \"title\": \"u probability description\", \"type\": \"nominal\"}], \"x\": {\"axis\": {\"title\": \"Proportion of record comparisons\"}, \"field\": \"m_probability\", \"type\": \"quantitative\"}, \"y\": {\"axis\": {\"title\": null}, \"field\": \"label_for_charts\", \"sort\": {\"field\": \"comparison_vector_value\", \"order\": \"descending\"}, \"type\": \"nominal\"}}, \"height\": {\"step\": 12}, \"resolve\": {\"scale\": {\"y\": \"independent\"}}, \"title\": {\"text\": \"Amongst matching record comparisons:\", \"fontSize\": 12, \"fontWeight\": \"bold\"}, \"transform\": [{\"filter\": \"(datum.bayes_factor != 'no-op filter due to vega lite issue 4680')\"}], \"width\": 150}, {\"mark\": \"bar\", \"encoding\": {\"color\": {\"value\": \"red\"}, \"row\": {\"field\": \"comparison_name\", \"header\": {\"labels\": false}, \"sort\": {\"field\": \"comparison_sort_order\"}, \"type\": \"nominal\"}, \"tooltip\": [{\"field\": \"comparison_name\", \"title\": \"Comparison name\", \"type\": \"nominal\"}, {\"field\": \"label_for_charts\", \"title\": \"Label\", \"type\": \"ordinal\"}, {\"field\": \"sql_condition\", \"title\": \"SQL condition\", \"type\": \"nominal\"}, {\"field\": \"m_probability\", \"format\": \".10~g\", \"title\": \"M probability\", \"type\": \"quantitative\"}, {\"field\": \"u_probability\", \"format\": \".10~g\", \"title\": \"U probability\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor\", \"format\": \",.6f\", \"title\": \"Bayes factor = m/u\", \"type\": \"quantitative\"}, {\"field\": \"log2_bayes_factor\", \"format\": \".4~g\", \"title\": \"Match weight = log2(m/u)\", \"type\": \"quantitative\"}, {\"field\": \"bayes_factor_description\", \"title\": \"Match weight description\", \"type\": \"nominal\"}, {\"field\": \"m_probability_description\", \"title\": \"m probability description\", \"type\": \"nominal\"}, {\"field\": \"u_probability_description\", \"title\": \"u probability description\", \"type\": \"nominal\"}], \"x\": {\"axis\": {\"title\": \"Proportion of record comparisons\"}, \"field\": \"u_probability\", \"type\": \"quantitative\"}, \"y\": {\"axis\": {\"title\": null}, \"field\": \"label_for_charts\", \"sort\": {\"field\": \"comparison_vector_value\", \"order\": \"descending\"}, \"type\": \"nominal\"}}, \"height\": {\"step\": 12}, \"resolve\": {\"scale\": {\"y\": \"independent\"}}, \"title\": {\"text\": \"Amongst non-matching record comparisons:\", \"fontSize\": 12, \"fontWeight\": \"bold\"}, \"transform\": [{\"filter\": \"(datum.bayes_factor != 'no-op filter2 due to vega lite issue 4680')\"}], \"width\": 150}], \"data\": {\"name\": \"data-0317940c67660b1599eba58da0e7c816\"}, \"title\": {\"text\": \"Proportion of record comparisons in each comparison level by match status\", \"subtitle\": \"(m and u probabilities)\"}, \"$schema\": \"https://vega.github.io/schema/vega-lite/v5.9.3.json\", \"datasets\": {\"data-0317940c67660b1599eba58da0e7c816\": [{\"comparison_name\": \"first_name\", \"sql_condition\": \"\\\"first_name_l\\\" = \\\"first_name_r\\\"\", \"label_for_charts\": \"Exact match on first_name\", \"m_probability\": 0.49136441060423774, \"u_probability\": 0.0057935713975033705, \"m_probability_description\": \"Amongst matching record comparisons, 49.14% of records (i.e. one in 2.035) are in the exact match on first_name comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.5794% of records (i.e. one in 173) are in the exact match on first_name comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 84.81200573725248, \"log2_bayes_factor\": 6.406196597784454, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on first_name` then comparison is 84.81 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"jaro_winkler_similarity(\\\"first_name_l\\\", \\\"first_name_r\\\") >= 0.9\", \"label_for_charts\": \"Jaro-Winkler distance of first_name >= 0.9\", \"m_probability\": 0.19147351818420014, \"u_probability\": 0.003386832528670639, \"m_probability_description\": \"Amongst matching record comparisons, 19.15% of records (i.e. one in 5.223) are in the jaro-winkler distance of first_name >= 0.9 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.3387% of records (i.e. one in 295) are in the jaro-winkler distance of first_name >= 0.9 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 56.53468737037173, \"log2_bayes_factor\": 5.821064412712824, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of first_name >= 0.9` then comparison is 56.53 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"jaro_winkler_similarity(\\\"first_name_l\\\", \\\"first_name_r\\\") >= 0.7\", \"label_for_charts\": \"Jaro-Winkler distance of first_name >= 0.7\", \"m_probability\": 0.11346321825427941, \"u_probability\": 0.019439490815246544, \"m_probability_description\": \"Amongst matching record comparisons, 11.35% of records (i.e. one in 8.813) are in the jaro-winkler distance of first_name >= 0.7 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 1.944% of records (i.e. one in 51.44) are in the jaro-winkler distance of first_name >= 0.7 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 5.836738180677517, \"log2_bayes_factor\": 2.5451623545127906, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of first_name >= 0.7` then comparison is 5.837 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"first_name\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.20369885295728274, \"u_probability\": 0.9713801052585794, \"m_probability_description\": \"Amongst matching record comparisons, 20.37% of records (i.e. one in 4.909) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 97.14% of records (i.e. one in 1.029) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.20970045799224857, \"log2_bayes_factor\": -2.253598082559084, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 4.769 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 0}, {\"comparison_name\": \"surname\", \"sql_condition\": \"\\\"surname_l\\\" = \\\"surname_r\\\"\", \"label_for_charts\": \"Exact match on surname\", \"m_probability\": 0.4345244833248351, \"u_probability\": 0.004889975550122249, \"m_probability_description\": \"Amongst matching record comparisons, 43.45% of records (i.e. one in 2.301) are in the exact match on surname comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.489% of records (i.e. one in 204) are in the exact match on surname comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 88.86025683992878, \"log2_bayes_factor\": 6.473466406178517, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `exact match on surname` then comparison is 88.86 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"surname\", \"sql_condition\": \"jaro_similarity(\\\"surname_l\\\", \\\"surname_r\\\") >= 0.9\", \"label_for_charts\": \"Jaro distance of 'surname >= 0.9'\", \"m_probability\": 0.2163740833960705, \"u_probability\": 0.0025524597651737017, \"m_probability_description\": \"Amongst matching record comparisons, 21.64% of records (i.e. one in 4.622) are in the jaro distance of 'surname >= 0.9' comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.2552% of records (i.e. one in 392) are in the jaro distance of 'surname >= 0.9' comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 84.77081063071945, \"log2_bayes_factor\": 6.405495677982012, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro distance of 'surname >= 0.9'` then comparison is 84.77 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"surname\", \"sql_condition\": \"jaro_similarity(\\\"surname_l\\\", \\\"surname_r\\\") >= 0.7\", \"label_for_charts\": \"Jaro distance of 'surname >= 0.7'\", \"m_probability\": 0.13100004234324159, \"u_probability\": 0.01614766651441468, \"m_probability_description\": \"Amongst matching record comparisons, 13.1% of records (i.e. one in 7.634) are in the jaro distance of 'surname >= 0.7' comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 1.615% of records (i.e. one in 61.93) are in the jaro distance of 'surname >= 0.7' comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 8.112629910105007, \"log2_bayes_factor\": 3.0201696756280945, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `jaro distance of 'surname >= 0.7'` then comparison is 8.113 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"surname\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.21810139093585287, \"u_probability\": 0.9764098981702893, \"m_probability_description\": \"Amongst matching record comparisons, 21.81% of records (i.e. one in 4.585) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 97.64% of records (i.e. one in 1.024) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.22337072918305792, \"log2_bayes_factor\": -2.162487949483275, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 3, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 4.477 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 1}, {\"comparison_name\": \"dob\", \"sql_condition\": \"\\\"dob_l\\\" = \\\"dob_r\\\"\", \"label_for_charts\": \"Exact match on date of birth\", \"m_probability\": 0.389983939276255, \"u_probability\": 0.0017477477477477479, \"m_probability_description\": \"Amongst matching record comparisons, 39% of records (i.e. one in 2.564) are in the exact match on date of birth comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.1748% of records (i.e. one in 572) are in the exact match on date of birth comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 223.13514051373352, \"log2_bayes_factor\": 7.801773924569989, \"comparison_vector_value\": 4, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `exact match on date of birth` then comparison is 223 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"damerau_levenshtein(\\\"dob_l\\\", \\\"dob_r\\\") <= 1\", \"label_for_charts\": \"DamerauLevenshtein distance <= 1\", \"m_probability\": 0.14884650150455297, \"u_probability\": 0.0016436436436436436, \"m_probability_description\": \"Amongst matching record comparisons, 14.88% of records (i.e. one in 6.718) are in the dameraulevenshtein distance <= 1 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.1644% of records (i.e. one in 608) are in the dameraulevenshtein distance <= 1 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 90.55886419186872, \"log2_bayes_factor\": 6.500783958589023, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `dameraulevenshtein distance <= 1` then comparison is 90.56 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"ABS(EPOCH(try_strptime(\\\"dob_l\\\", '%Y-%m-%d')) - EPOCH(try_strptime(\\\"dob_r\\\", '%Y-%m-%d'))) <= 31557600.0\", \"label_for_charts\": \"Abs date difference <= 1 year\", \"m_probability\": 0.19399707416347944, \"u_probability\": 0.03546146146146146, \"m_probability_description\": \"Amongst matching record comparisons, 19.4% of records (i.e. one in 5.155) are in the abs date difference <= 1 year comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 3.546% of records (i.e. one in 28.2) are in the abs date difference <= 1 year comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 5.470645206608592, \"log2_bayes_factor\": 2.4517109941661124, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `abs date difference <= 1 year` then comparison is 5.471 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"ABS(EPOCH(try_strptime(\\\"dob_l\\\", '%Y-%m-%d')) - EPOCH(try_strptime(\\\"dob_r\\\", '%Y-%m-%d'))) <= 2629800.0\", \"label_for_charts\": \"Abs date difference <= 1 month\", \"m_probability\": 0.012500000000000011, \"u_probability\": 0.06299605249474372, \"m_probability_description\": \"Amongst matching record comparisons, 1.25% of records (i.e. one in 80) are in the abs date difference <= 1 month comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 6.3% of records (i.e. one in 15.87) are in the abs date difference <= 1 month comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.19842513149602492, \"log2_bayes_factor\": -2.3333333333333335, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `abs date difference <= 1 month` then comparison is 5.04 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"dob\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.26717248505571256, \"u_probability\": 0.9611471471471471, \"m_probability_description\": \"Amongst matching record comparisons, 26.72% of records (i.e. one in 3.743) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 96.11% of records (i.e. one in 1.04) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.2779725100872715, \"log2_bayes_factor\": -1.846985879283028, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 3.597 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 2}, {\"comparison_name\": \"city\", \"sql_condition\": \"\\\"city_l\\\" = \\\"city_r\\\"\", \"label_for_charts\": \"Exact match on city\", \"m_probability\": 0.5611221596314007, \"u_probability\": 0.0551475711801453, \"m_probability_description\": \"Amongst matching record comparisons, 56.11% of records (i.e. one in 1.782) are in the exact match on city comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 5.515% of records (i.e. one in 18.13) are in the exact match on city comparison level\", \"has_tf_adjustments\": true, \"tf_adjustment_column\": \"city\", \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 10.17492062884214, \"log2_bayes_factor\": 3.34694563544056, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 1, \"bayes_factor_description\": \"If comparison level is `exact match on city` then comparison is 10.17 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 3}, {\"comparison_name\": \"city\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.4388778403685992, \"u_probability\": 0.9448524288198547, \"m_probability_description\": \"Amongst matching record comparisons, 43.89% of records (i.e. one in 2.279) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 94.49% of records (i.e. one in 1.058) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.4644935303989948, \"log2_bayes_factor\": -1.1062695924156825, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 1, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 2.153 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 3}, {\"comparison_name\": \"email\", \"sql_condition\": \"\\\"email_l\\\" = \\\"email_r\\\"\", \"label_for_charts\": \"Exact match on email\", \"m_probability\": 0.552153409103165, \"u_probability\": 0.0021938713143283602, \"m_probability_description\": \"Amongst matching record comparisons, 55.22% of records (i.e. one in 1.811) are in the exact match on email comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.2194% of records (i.e. one in 456) are in the exact match on email comparison level\", \"has_tf_adjustments\": true, \"tf_adjustment_column\": \"email\", \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 251.67994380390687, \"log2_bayes_factor\": 7.975446443510344, \"comparison_vector_value\": 4, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `exact match on email` then comparison is 252 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"NULLIF(regexp_extract(\\\"email_l\\\", '^[^@]+', 0), '') = NULLIF(regexp_extract(\\\"email_r\\\", '^[^@]+', 0), '')\", \"label_for_charts\": \"Exact match on username\", \"m_probability\": 0.22055262276218585, \"u_probability\": 0.0010390328952024346, \"m_probability_description\": \"Amongst matching record comparisons, 22.06% of records (i.e. one in 4.534) are in the exact match on username comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.1039% of records (i.e. one in 962) are in the exact match on username comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 212.2672186612683, \"log2_bayes_factor\": 7.72973777662564, \"comparison_vector_value\": 3, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `exact match on username` then comparison is 212 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"jaro_winkler_similarity(\\\"email_l\\\", \\\"email_r\\\") >= 0.88\", \"label_for_charts\": \"Jaro-Winkler distance of email >= 0.88\", \"m_probability\": 0.2138389403844258, \"u_probability\": 0.0009135769109519858, \"m_probability_description\": \"Amongst matching record comparisons, 21.38% of records (i.e. one in 4.676) are in the jaro-winkler distance of email >= 0.88 comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.09136% of records (i.e. one in 1,095) are in the jaro-winkler distance of email >= 0.88 comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 234.0678029631863, \"log2_bayes_factor\": 7.870782688940884, \"comparison_vector_value\": 2, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `jaro-winkler distance of email >= 0.88` then comparison is 234 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"jaro_winkler_similarity(NULLIF(regexp_extract(\\\"email_l\\\", '^[^@]+', 0), ''), NULLIF(regexp_extract(\\\"email_r\\\", '^[^@]+', 0), '')) >= 0.88\", \"label_for_charts\": \"Jaro-Winkler >0.88 on username\", \"m_probability\": 0.012500000000000011, \"u_probability\": 0.000501823937001795, \"m_probability_description\": \"Amongst matching record comparisons, 1.25% of records (i.e. one in 80) are in the jaro-winkler >0.88 on username comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 0.05018% of records (i.e. one in 1,993) are in the jaro-winkler >0.88 on username comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 24.909134615384634, \"log2_bayes_factor\": 4.638602995720225, \"comparison_vector_value\": 1, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `jaro-winkler >0.88 on username` then comparison is 24.91 times more likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}, {\"comparison_name\": \"email\", \"sql_condition\": \"ELSE\", \"label_for_charts\": \"All other comparisons\", \"m_probability\": 0.01345502775022321, \"u_probability\": 0.9953516949425154, \"m_probability_description\": \"Amongst matching record comparisons, 1.346% of records (i.e. one in 74.32) are in the all other comparisons comparison level\", \"u_probability_description\": \"Amongst non-matching record comparisons, 99.54% of records (i.e. one in 1.005) are in the all other comparisons comparison level\", \"has_tf_adjustments\": false, \"tf_adjustment_column\": null, \"tf_adjustment_weight\": 1.0, \"is_null_level\": false, \"bayes_factor\": 0.013517862900710968, \"log2_bayes_factor\": -6.2089891022264005, \"comparison_vector_value\": 0, \"max_comparison_vector_value\": 4, \"bayes_factor_description\": \"If comparison level is `all other comparisons` then comparison is 73.98 times less likely to be a match\", \"probability_two_random_records_match\": 0.0001, \"comparison_sort_order\": 4}]}}, {\"mode\": \"vega-lite\"});\n",
              "</script>"
            ],
            "text/plain": [
              "alt.HConcatChart(...)"
            ]
          },
          "execution_count": 1,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "import splink.comparison_library as cl\n",
        "\n",
        "from splink import DuckDBAPI, Linker, SettingsCreator, block_on, splink_datasets\n",
        "\n",
        "df = splink_datasets.fake_1000\n",
        "\n",
        "settings = SettingsCreator(\n",
        "    link_type=\"dedupe_only\",\n",
        "    comparisons=[\n",
        "        cl.JaroWinklerAtThresholds(\"first_name\", [0.9, 0.7]),\n",
        "        cl.JaroAtThresholds(\"surname\", [0.9, 0.7]),\n",
        "        cl.DateOfBirthComparison(\n",
        "            \"dob\",\n",
        "            input_is_string=True,\n",
        "            datetime_metrics=[\"year\", \"month\"],\n",
        "            datetime_thresholds=[1, 1],\n",
        "        ),\n",
        "        cl.ExactMatch(\"city\").configure(term_frequency_adjustments=True),\n",
        "        cl.EmailComparison(\"email\"),\n",
        "    ],\n",
        "    blocking_rules_to_generate_predictions=[\n",
        "        block_on(\"first_name\"),\n",
        "        block_on(\"surname\"),\n",
        "    ],\n",
        ")\n",
        "\n",
        "linker = Linker(df, settings, DuckDBAPI())\n",
        "linker.training.estimate_u_using_random_sampling(max_pairs=1e6)\n",
        "\n",
        "blocking_rule_for_training = block_on(\"first_name\", \"surname\")\n",
        "\n",
        "linker.training.estimate_parameters_using_expectation_maximisation(\n",
        "    blocking_rule_for_training\n",
        ")\n",
        "\n",
        "blocking_rule_for_training = block_on(\"dob\")\n",
        "linker.training.estimate_parameters_using_expectation_maximisation(\n",
        "    blocking_rule_for_training\n",
        ")\n",
        "\n",
        "chart = linker.visualisations.m_u_parameters_chart()\n",
        "chart\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "base",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.8"
    },
    "orig_nbformat": 4
  },
  "nbformat": 4,
  "nbformat_minor": 2
}
